# coding: utf-8

import bs4
from scrapy.contrib.spiders.crawl import CrawlSpider, Rule
import scrapy
from ..items import GoufangItem


class GoufangSpider(CrawlSpider):

    name = 'goufang2spider'
    allowed_domains = ['goufang.com']
    start_urls = ['http://goufang.com/']

    def parse(self, response):
        data = '''      <ul>
        <li><a href="http://qd.goufang.com/"  class="qingdao">
          青岛站
          </a></li>
        <li><a href="http://gz.goufang.com/">广州站</a></li>
        <li><a href="http://sz.goufang.com/">深圳站</a></li>

      </ul>'''
        soup = bs4.BeautifulSoup(data, 'lxml')
        citybox = soup.select('li > a')
        for child in citybox:
            city = child.get_text().replace('站', '').strip()
            city_url = child.get('href')
            city_id = child.get('href').replace('http://', '').replace('.goufang.com/', '')
            site_url = 'http://house.' + city_id + '.goufang.com/'
            citys = {
                'website': '购房者', 'web_url': 'goufang.com',
                'city': city, 'city_id': city_id, 'city_url': city_url
            }
            print(site_url)
            yield scrapy.Request(site_url, callback=self.parse_city_area, meta=citys)

    def parse_city_area(self, response):
        meta = response.meta
        data = response.body
        soup = bs4.BeautifulSoup(data, 'lxml')
        areabox = soup.find_all('ul', class_='nav-category-content')[0]
        areas = areabox.select('li > a')
        for child in areas:
            area = child.get_text()
            if area != '不限':
                area_url = child.get('href')
                print(area, area_url)
                meta['area'] = area

                yield scrapy.Request(area_url, callback=self.parse_city_estate, meta=meta)

    def parse_city_estate(self, response):
        meta = response.meta
        city_url = meta['city_url']
        city_id = meta['city_id']
        area = meta['area']

        data = response.body
        soup = bs4.BeautifulSoup(data, 'lxml')
        estatebox = soup.find_all('h3', class_='xp_lm')
        for child in estatebox:
            estate = child.find_all('a')[0].get_text()
            estate_url = child.find_all('a')[0].get('href')
            estate_id = child.find_all('a')[0].get('href').replace('http://house.' + city_id + '.goufang.com/',
                                                                   '').replace('.html', '')
            # print(area, estate, estate_id, estate_url)
            item = GoufangItem()
            item['website'] = meta['website']
            item['web_url'] = meta['web_url']
            item['city'] = meta['city']
            item['city_id'] = meta['city_id']
            item['area'] = area
            item['estate'] = estate
            item['estate_id'] = estate_id
            item['estate_url'] = estate_url
            yield item

        pages = soup.select('.pagebtn > a')
        if pages:
            next_pages = pages[-1].get_text()
            if next_pages == '下一页':
                next_url = pages[-1].get('href')
                yield scrapy.Request(next_url, callback=self.parse_city_estate, meta=meta)
